From: Matthieu Gallien <matthieu.gallien@nextcloud.com>
Date: Wed, 26 Mar 2025 15:06:53 +0000 (+0100)
Subject: use maximum chunk size to decide the size of a bulk upload request
X-Git-Tag: archive/raspbian/3.16.7-1_deb13u1+rpi1~1^2~12^2^2~37^2~5
X-Git-Url: https://dgit.raspbian.org/%22http://www.example.com/cgi/success/%22http:/www.example.com/cgi/success?a=commitdiff_plain;h=b0067c398877c3639ecdb52e975c99275b56fd71;p=nextcloud-desktop.git

use maximum chunk size to decide the size of a bulk upload request

Signed-off-by: Matthieu Gallien <matthieu.gallien@nextcloud.com>
---

diff --git a/src/libsync/bulkpropagatorjob.cpp b/src/libsync/bulkpropagatorjob.cpp
index 9513b1711..c03a82324 100644
--- a/src/libsync/bulkpropagatorjob.cpp
+++ b/src/libsync/bulkpropagatorjob.cpp
@@ -58,7 +58,6 @@ QByteArray getHeaderFromJsonReply(const QJsonObject &reply, const QByteArray &he
     return reply.value(headerName).toString().toLatin1();
 }
 
-constexpr auto batchSize = 100;
 constexpr auto parallelJobsMaximumCount = 1;
 
 }
@@ -70,10 +69,10 @@ Q_LOGGING_CATEGORY(lcBulkPropagatorJob, "nextcloud.sync.propagator.bulkupload",
 BulkPropagatorJob::BulkPropagatorJob(OwncloudPropagator *propagator, const std::deque<SyncFileItemPtr> &items)
     : PropagatorJob(propagator)
     , _items(items)
-    , _currentBatchSize(batchSize)
+    , _currentBatchSize(_items.size())
 {
-    _filesToUpload.reserve(batchSize);
-    _pendingChecksumFiles.reserve(batchSize);
+    _filesToUpload.reserve(_items.size());
+    _pendingChecksumFiles.reserve(_items.size());
 }
 
 bool BulkPropagatorJob::scheduleSelfOrChild()
@@ -84,11 +83,15 @@ bool BulkPropagatorJob::scheduleSelfOrChild()
 
     _state = Running;
 
-    for(auto i = 0; i < _currentBatchSize && !_items.empty(); ++i) {
+    qCDebug(lcBulkPropagatorJob()) << "max chunk size" << PropagatorJob::propagator()->syncOptions().maxChunkSize();
+
+    for(auto batchDataSize = 0; batchDataSize <= PropagatorJob::propagator()->syncOptions().maxChunkSize() && !_items.empty(); ) {
         const auto currentItem = _items.front();
         _items.pop_front();
         _pendingChecksumFiles.insert(currentItem->_file);
 
+        batchDataSize += currentItem->_size;
+
         QMetaObject::invokeMethod(this, [this, currentItem] {
             UploadFileInfo fileToUpload;
             fileToUpload._file = currentItem->_file;
@@ -117,7 +120,7 @@ bool BulkPropagatorJob::handleBatchSize()
     }
 
     // change batch size before trying it again
-    const auto halfBatchSize = batchSize / 2;
+    const auto halfBatchSize = static_cast<int>(_items.size() / 2);
 
     // we already tried to upload with half of the batch size
     if(_currentBatchSize == halfBatchSize) {
diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp
index 1efd01e5d..8f875cc4d 100644
--- a/src/libsync/configfile.cpp
+++ b/src/libsync/configfile.cpp
@@ -270,13 +270,13 @@ qint64 ConfigFile::chunkSize() const
 qint64 ConfigFile::maxChunkSize() const
 {
     QSettings settings(configFile(), QSettings::IniFormat);
-    return settings.value(QLatin1String(maxChunkSizeC), 5LL * 1000LL * 1000LL * 1000LL).toLongLong(); // default to 5000 MB
+    return settings.value(QLatin1String(maxChunkSizeC), 100LL * 1024LL * 1024LL).toLongLong(); // default to 100 MiB
 }
 
 qint64 ConfigFile::minChunkSize() const
 {
     QSettings settings(configFile(), QSettings::IniFormat);
-    return settings.value(QLatin1String(minChunkSizeC), 5LL * 1000LL * 1000LL).toLongLong(); // default to 5 MB
+    return settings.value(QLatin1String(minChunkSizeC), 5LL * 1024LL * 1024LL).toLongLong(); // default to 5 MiB
 }
 
 chrono::milliseconds ConfigFile::targetChunkUploadDuration() const
diff --git a/test/testsyncengine.cpp b/test/testsyncengine.cpp
index 26daf2c5f..21ca008f9 100644
--- a/test/testsyncengine.cpp
+++ b/test/testsyncengine.cpp
@@ -1208,7 +1208,7 @@ private slots:
 
         QVERIFY(fakeFolder.syncOnce());
         QCOMPARE(nPUT, 0);
-        QCOMPARE(nPOST, 2);
+        QCOMPARE(nPOST, 1);
         nPUT = 0;
         nPOST = 0;
 
@@ -1219,7 +1219,7 @@ private slots:
 
         QVERIFY(!fakeFolder.syncOnce());
         QCOMPARE(nPUT, 120);
-        QCOMPARE(nPOST, 2);
+        QCOMPARE(nPOST, 1);
         nPUT = 0;
         nPOST = 0;